package com.shujia.core

import org.apache.spark.{SparkConf, SparkContext}

object Demo16Conf {
  def main(args: Array[String]): Unit = {
    //1、创建spark配置文件对象
    val conf = new SparkConf()
    // 运行方式
    conf.setMaster("local") //本地运行
    conf.setAppName("WordCount") //程序名


    conf.set("spark.shuffle.file.buffer","64k")
    conf.set("spark.reducer.maxSizeInFlight","96M")
    conf.set("spark.shuffle.io.maxRetries","5")
    conf.set("spark.shuffle.io.retryWait","30s")


    //一般同时调节
    conf.set("spark.shuffle.memoryFraction","0.4")//默认0.2
    conf.set("spark.storage.memoryFraction","0.4")//默认0.6
    //还有0.2的内存用于运行时

    //如果不希望数据排序，可以调大一点
    conf.set("spark.shuffle.sort.bypassMergeThreshold","1000")







    //2、创建spark上下文对象
    val sc = new SparkContext(conf)

    val studentRDD = sc.textFile("data/student.txt")

  }

}
